{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning\n",
      "999\n",
      "Accuracy: 0.6256256256256256\n",
      "Accuracy: 0.6256256256256256\n",
      "F1 score: 0.6056783552985715\n",
      "Macro F1: 0.6056783552985715\n",
      "Micro F1: 0.6256256256256256\n",
      "                        precision    recall  f1-score   support\n",
      "\n",
      "              database    0.52900   0.70154   0.60317       325\n",
      "wireless communication    0.98182   0.33645   0.50116       321\n",
      "           data mining    0.63100   0.81870   0.71270       353\n",
      "\n",
      "              accuracy                        0.62563       999\n",
      "             macro avg    0.71394   0.61889   0.60568       999\n",
      "          weighted avg    0.71054   0.62563   0.60910       999\n",
      "\n",
      "AUC score for each class: [0.7001757589591417, 0.6667493728117332, 0.7785434883659741]\n",
      "Average AUC OvR score: 0.7151562067122829\n",
      "AUC score for each pair: []\n",
      "Average AUC OvO score: nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/hpc2hdd/home/xzou428/miniconda3/envs/yuh/lib/python3.10/site-packages/IPython/core/magics/osm.py:417: UserWarning: This is now an optional IPython functionality, setting dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n",
      "/tmp/ipykernel_2309458/2099491970.py:120: DeprecationWarning: Calling nonzero on 0d arrays is deprecated, as it behaves surprisingly. Use `atleast_1d(cond).nonzero()` if the old behavior was intended. If the context of this warning is of the form `arr[nonzero(cond)]`, just use `arr[cond]`.\n",
      "  idx = np.where((labels == class1) | (labels == class2))[0]\n"
     ]
    }
   ],
   "source": [
    "%cd /hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning/\n",
    "\n",
    "import json\n",
    "import os.path as osp\n",
    "import os\n",
    "import torch as th\n",
    "import re\n",
    "import pandas as pd\n",
    "from tqdm import tqdm \n",
    "from sklearn.metrics import classification_report, roc_auc_score\n",
    "from sklearn.preprocessing import label_binarize\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from itertools import combinations\n",
    "\n",
    "\n",
    "res_dict = {}\n",
    "for file in Path(\"checkpoints/higpt-stage2-llama3-IMDB_Few_Shot-epoch30-8192-full_finetune/lightning_logs/version_2/predict_eval_ACM_1000_20240802_19\").glob(\"*.txt\"):\n",
    "    res_dict[file.stem.strip(\"[]\").strip(\"'\")] = file.read_text().strip()\n",
    "\n",
    "# for file in Path(\"checkpoints/higpt-stage2-llama3-IMDB_Few_Shot-epoch30-8192-full_finetune/lightning_logs/version_1/predict_20240802_152618\").glob(\"*.txt\"):\n",
    "#     res_dict[file.stem.strip(\"[]\").strip(\"'\")] = file.read_text().strip()\n",
    "# assert len(res_dict) == 999\n",
    "\n",
    "ground_truth = json.load(open(\"dataset/ACM_test_1000/test_cot_prompts_higpt_with_skg.json\"))\n",
    "ground_truth = {item['id']: item for item in ground_truth}\n",
    "\n",
    "class_map = {'database': 0, 'wireless communication': 1, 'data mining': 2}\n",
    "\n",
    "pattern = r'(Database|Wireless Communication|Data Mining)'\n",
    "\n",
    "\n",
    "correct = 0\n",
    "total = len(res_dict)\n",
    "\n",
    "trues = []\n",
    "preds = []\n",
    "\n",
    "for nid in res_dict: \n",
    "    gpt_res = res_dict[nid]\n",
    "    truth_item = ground_truth[nid]\n",
    "\n",
    "    matches = re.search(pattern, gpt_res, re.I)\n",
    "    # matched_label = matches.group(1) if matches else 'database'\n",
    "    if matches: \n",
    "        matched_label = matches.group(1)\n",
    "    else: \n",
    "        matched_label = 'Database'\n",
    "    matched_label = matched_label.lower()\n",
    "\n",
    "    true_y = truth_item['graph']['label']\n",
    "\n",
    "    pred_y = class_map[matched_label]\n",
    "\n",
    "    trues.append(true_y)\n",
    "    preds.append(pred_y)\n",
    "\n",
    "    if true_y == pred_y:\n",
    "        correct = correct + 1\n",
    "        # print(\"Correct:\", nid, matched_label, gpt_res, true_y)\n",
    "    else:\n",
    "        # print(\"Incorrect:\", nid, matched_label, gpt_res, true_y)\n",
    "        pass\n",
    "\n",
    "acc = correct / total\n",
    "\n",
    "print(total)\n",
    "print(\"Accuracy:\", acc)\n",
    "\n",
    "num_classes = 3\n",
    "confusion_matrix = np.zeros((num_classes, num_classes))\n",
    "\n",
    "for true_label, predicted_label in zip(trues, preds):\n",
    "    confusion_matrix[true_label, predicted_label] += 1\n",
    "\n",
    "# Accuracy\n",
    "accuracy = np.trace(confusion_matrix) / np.sum(confusion_matrix)\n",
    "\n",
    "# F1-score and Macro-F1\n",
    "precision = np.diag(confusion_matrix) / np.sum(confusion_matrix, axis=1)\n",
    "recall = np.diag(confusion_matrix) / np.sum(confusion_matrix, axis=0)\n",
    "f1_score = 2 * precision * recall / (precision + recall)\n",
    "macro_f1 = np.mean(f1_score)\n",
    "\n",
    "micro_precision = np.sum(np.diag(confusion_matrix)) / np.sum(confusion_matrix)\n",
    "micro_recall = np.sum(np.diag(confusion_matrix)) / np.sum(confusion_matrix)\n",
    "micro_f1 = 2 * micro_precision * micro_recall / (micro_precision + micro_recall)\n",
    "\n",
    "print(\"Accuracy:\", accuracy)\n",
    "print(\"F1 score:\", f1_score.mean())\n",
    "print(\"Macro F1:\", macro_f1)\n",
    "print(\"Micro F1:\", micro_f1)\n",
    "\n",
    "print(classification_report(trues, preds, target_names=list(class_map.keys()), digits=5))\n",
    "# print(\"AUC-ROC:\", roc_auc_score(trues, preds, multi_class='ovo', average='weighted'))\n",
    "\n",
    "# Binarize the labels and predictions\n",
    "labels_binarized = label_binarize(trues, classes=[0, 1, 2])\n",
    "preds_binarized = label_binarize(preds, classes=[0, 1, 2])\n",
    "\n",
    "# Calculate the AUC score for each class\n",
    "auc_scores = []\n",
    "for i in range(num_classes):\n",
    "    auc = roc_auc_score(labels_binarized[:, i], preds_binarized[:, i])\n",
    "    auc_scores.append(auc)\n",
    "\n",
    "# Calculate the average AUC score\n",
    "average_auc = np.mean(auc_scores)\n",
    "\n",
    "print(f\"AUC score for each class: {auc_scores}\")\n",
    "print(f\"Average AUC OvR score: {average_auc}\")\n",
    "\n",
    "\n",
    "# # Initialize a list to store AUC scores\n",
    "# auc_scores = []\n",
    "# labels = trues\n",
    "# # Iterate through all pairs of classes\n",
    "# for (class1, class2) in combinations(range(num_classes), 2):\n",
    "#     # Extract indices for the current pair\n",
    "#     idx = np.where((labels == class1) | (labels == class2))[0]\n",
    "    \n",
    "#     # Ensure both classes are present\n",
    "#     if len(idx) == 0 or len(np.unique(labels[idx])) < 2:\n",
    "#         continue\n",
    "    \n",
    "#     # Filter predictions and labels for the current pair\n",
    "#     preds_pair = preds[idx]\n",
    "#     labels_pair = labels[idx]\n",
    "    \n",
    "#     # Binarize labels for the current pair\n",
    "#     labels_pair = np.where(labels_pair == class1, 0, 1)\n",
    "#     preds_pair = np.where(preds_pair == class1, 0, 1)\n",
    "    \n",
    "#     # Calculate the AUC score for the current pair\n",
    "#     auc = roc_auc_score(labels_pair, preds_pair)\n",
    "#     auc_scores.append(auc)\n",
    "\n",
    "# # Calculate the average AUC score\n",
    "# if auc_scores:  # Check if auc_scores is not empty\n",
    "#     average_auc = np.mean(auc_scores)\n",
    "# else:\n",
    "#     average_auc = np.nan  # or any other value indicating no valid pairs were found\n",
    "\n",
    "# print(f\"AUC score for each pair: {auc_scores}\")\n",
    "# print(f\"Average AUC OvO score: {average_auc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning\n",
      "942\n",
      "Accuracy: 0.5583864118895966\n",
      "Accuracy: 0.5583864118895966\n",
      "F1 score: 0.5450397147666147\n",
      "Macro F1: 0.5450397147666147\n",
      "Micro F1: 0.5583864118895966\n"
     ]
    }
   ],
   "source": [
    "%cd /hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning/\n",
    "\n",
    "import json\n",
    "import os.path as osp\n",
    "import os\n",
    "import torch as th\n",
    "import re\n",
    "import pandas as pd\n",
    "from tqdm import tqdm \n",
    "from sklearn.metrics import classification_report\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "\n",
    "\n",
    "res_dict = {}\n",
    "for file in Path(\"checkpoints/higpt-stage2-llama3-MIX-epoch30-8192-full_finetune/lightning_logs/version_2/predict_eval_ACM_1000_20240802_162401\").glob(\"*.txt\"):\n",
    "    res_dict[file.stem.strip(\"[]\").strip(\"'\")] = file.read_text().strip()\n",
    "\n",
    "for file in Path(\"checkpoints/higpt-stage2-llama3-MIX-epoch30-8192-full_finetune/lightning_logs/version_2/predict_eval_ACM_1000_20240802_162853\").glob(\"*.txt\"):\n",
    "    res_dict[file.stem.strip(\"[]\").strip(\"'\")] = file.read_text().strip()\n",
    "# assert len(res_dict) == 999\n",
    "\n",
    "ground_truth = json.load(open(\"dataset/ACM_test_1000/test_cot_prompts_higpt_with_skg.json\"))\n",
    "ground_truth = {item['id']: item for item in ground_truth}\n",
    "\n",
    "class_map = {'database': 0, 'wireless communication': 1, 'data mining': 2}\n",
    "\n",
    "pattern = r'(Database|Wireless Communication|Data Mining)'\n",
    "\n",
    "\n",
    "correct = 0\n",
    "total = len(res_dict)\n",
    "\n",
    "trues = []\n",
    "preds = []\n",
    "\n",
    "for nid in res_dict: \n",
    "    gpt_res = res_dict[nid]\n",
    "    truth_item = ground_truth[nid]\n",
    "\n",
    "    matches = re.search(pattern, gpt_res, re.I)\n",
    "    # matched_label = matches.group(1) if matches else 'database'\n",
    "    if matches: \n",
    "        matched_label = matches.group(1)\n",
    "    else: \n",
    "        matched_label = 'Database'\n",
    "    matched_label = matched_label.lower()\n",
    "\n",
    "    true_y = truth_item['graph']['label']\n",
    "\n",
    "    pred_y = class_map[matched_label]\n",
    "\n",
    "    trues.append(true_y)\n",
    "    preds.append(pred_y)\n",
    "\n",
    "    if true_y == pred_y:\n",
    "        correct = correct + 1\n",
    "        # print(\"Correct:\", nid, matched_label, gpt_res, true_y)\n",
    "    else:\n",
    "        # print(\"Incorrect:\", nid, matched_label, gpt_res, true_y)\n",
    "        pass\n",
    "\n",
    "acc = correct / total\n",
    "\n",
    "print(total)\n",
    "print(\"Accuracy:\", acc)\n",
    "\n",
    "num_classes = 3\n",
    "confusion_matrix = np.zeros((num_classes, num_classes))\n",
    "\n",
    "for true_label, predicted_label in zip(trues, preds):\n",
    "    confusion_matrix[true_label, predicted_label] += 1\n",
    "\n",
    "# Accuracy\n",
    "accuracy = np.trace(confusion_matrix) / np.sum(confusion_matrix)\n",
    "\n",
    "# F1-score and Macro-F1\n",
    "precision = np.diag(confusion_matrix) / np.sum(confusion_matrix, axis=1)\n",
    "recall = np.diag(confusion_matrix) / np.sum(confusion_matrix, axis=0)\n",
    "f1_score = 2 * precision * recall / (precision + recall)\n",
    "macro_f1 = np.mean(f1_score)\n",
    "\n",
    "micro_precision = np.sum(np.diag(confusion_matrix)) / np.sum(confusion_matrix)\n",
    "micro_recall = np.sum(np.diag(confusion_matrix)) / np.sum(confusion_matrix)\n",
    "micro_f1 = 2 * micro_precision * micro_recall / (micro_precision + micro_recall)\n",
    "\n",
    "print(\"Accuracy:\", accuracy)\n",
    "print(\"F1 score:\", f1_score.mean())\n",
    "print(\"Macro F1:\", macro_f1)\n",
    "print(\"Micro F1:\", micro_f1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.14 ('yuh')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "92619f85c62eb73280c07ca2268c8e47b90999a589aa097a1a08a504bd2fb2c6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
